#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-10-06 16:32:26
# Project: toutiao_channel

from pyspider.libs.base_handler import *


class Handler(BaseHandler):
    crawl_config = {
    }


    @every(minutes=24 * 60)
    def on_start(self):
        #url = 'https://www.toutiao.com/a6608421255685079559/'
        #self.detail_page(url)
        url = 'https://www.toutiao.com/ch/news_tech/'
        self.crawl(url, fetch_type='js', callback=self.phantomjs_parser)

    def phantomjs_parser(self, response):
        for x in response.doc('.wcommonFeed .item-inner').items():
            arc = {
                "title": x('.title-box a').text(),
                "comment": ''.join([x for x in x('.comment').text() if x.isdigit()]),
                "url": x('.title-box a').attr.href,
                "author": x('.source').text().strip('\xa0⋅'),
                "author_logo": x('.media-avatar img').attr.src,
            }
            self.crawl(arc.get('url'), fetch_type='js', callback=self.detail_page, save=arc)


    @config(priority=2)
    def detail_page(self, response):
        arc = response.save
        arc['publish_time'] = response.doc('.article-sub span')[1].text
        arc['describe'] = response.doc('meta[name=description]').attr("content")
        arc['content'] = response.doc('.article-content').html()
        arc['img_url'] = response.doc('.pgc-img img').attr('src')
        print('arc', type(arc))
        return arc
